#!pip install seaborn

#!pip install scikit-learn

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import plotly.express as px
#choose plots style
sns.set_style('darkgrid')
#make sure plots are inline with the notebook
%matplotlib inline
from plotly.subplots import make_subplots
import plotly.graph_objects as go

import warnings
warnings.filterwarnings('ignore')

def add_assistance_percentage_per_indicator(df, indicator):
    # Define column name
    col_name = "Total "+indicator+" appointments"

    # Add a column for the total appointmens per indicator class to get the percentage
    df[col_name] = df[indicator].map(dict(df.groupby([indicator])["Assistance"].sum()))

    # Add a column representing the percentage of assistance to the apointment
    df["Percentage of appointments"] = (df["Assistance"] * 100 ) / df[col_name].astype(float)

    return df

def plot_missing_assistance_percentage_per_indicator(df, indicator, title):

    # Plot appointments missing rate by scholarship class
    days = ["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"]

    fig = px.bar(df[df["Assisted"] == False ],
                title =title,
                color=indicator, y="Percentage of appointments", x=indicator,
                labels = {
                    "Percentage of appointments" : "Percentage of missed appointments (%)"
                },
                category_orders={
                    indicator: days
                },
                text_auto=".4s")

    # Set values over the bars
    fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

    # Hide legend
    fig.update_layout(showlegend=False)

    fig.show()

import seaborn as sns

def plot_confusion_matrix(cm, title="Confusion Matrix"):
    plt.figure(figsize=(6,6))
    sns.set(font_scale=1.5)
    ax = sns.heatmap(cm, annot=True, annot_kws={"size":16}, fmt='g', cmap='Blues', cbar=False)

    ax.set_xlabel("Predicted values")
    ax.xaxis.set_ticklabels(['Negative','Positive'])

    ax.set_ylabel("Actual values")
    ax.yaxis.set_ticklabels(['Negative','Positive'])

    ax.set_title(title)

from sklearn.metrics import roc_curve, auc

def plot_roc_curve(x_test, y_test, model):

    y_score = model.predict_proba(x_test)[:, 1]

    fpr, tpr, thresholds = roc_curve(y_test, y_score)

    fig = px.area(
        x=fpr, y=tpr,
        title=f'ROC Curve (AUC={auc(fpr, tpr):.4f})',
        labels=dict(x='False Positive Rate', y='True Positive Rate'),
        width=700, height=500
    )

    fig.add_shape(
        type='line', line=dict(dash='dash'),
        x0=0, x1=1, y0=0, y1=1
    )

    fig.update_yaxes(scaleanchor="x", scaleratio=1)
    fig.update_xaxes(constrain='domain')
    fig.show()

df = pd.read_csv('./data/KaggleV2-May-2016.csv')
df.head()

# Get the shape and types of our data
print(df.shape)
pd.DataFrame(df.dtypes)

(110527, 14)

# Get some statistics about our data
df.describe()

# Show data basic info
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 110527 entries, 0 to 110526
Data columns (total 14 columns):
 #   Column          Non-Null Count   Dtype  
---  ------          --------------   -----  
 0   PatientId       110527 non-null  float64
 1   AppointmentID   110527 non-null  int64  
 2   Gender          110527 non-null  object 
 3   ScheduledDay    110527 non-null  object 
 4   AppointmentDay  110527 non-null  object 
 5   Age             110527 non-null  int64  
 6   Neighbourhood   110527 non-null  object 
 7   Scholarship     110527 non-null  int64  
 8   Hipertension    110527 non-null  int64  
 9   Diabetes        110527 non-null  int64  
 10  Alcoholism      110527 non-null  int64  
 11  Handcap         110527 non-null  int64  
 12  SMS_received    110527 non-null  int64  
 13  No-show         110527 non-null  object 
dtypes: float64(1), int64(8), object(5)
memory usage: 11.8+ MB

# Check for missing values in our data
df.isna().any()

PatientId         False
AppointmentID     False
Gender            False
ScheduledDay      False
AppointmentDay    False
Age               False
Neighbourhood     False
Scholarship       False
Hipertension      False
Diabetes          False
Alcoholism        False
Handcap           False
SMS_received      False
No-show           False
dtype: bool

# Check for duplicated rows in our data
df.duplicated().any()

False

df[(df["Age"] == -1)]

# Drop corrupted value
df.drop(df[(df["Age"] == -1)].index, axis=0, inplace=True)

df.describe()

# Drop irrelevant columns
df.drop(['PatientId','AppointmentID'],axis=1,inplace=True)
df.head()

# Rename column
df.rename(columns={"No-show": "Assisted"}, inplace=True)

# Change values in the column
df["Assisted"].replace("No", True, inplace=True)
df["Assisted"].replace("Yes", False, inplace=True)

df.head()

df.dtypes

Gender            object
ScheduledDay      object
AppointmentDay    object
Age                int64
Neighbourhood     object
Scholarship        int64
Hipertension       int64
Diabetes           int64
Alcoholism         int64
Handcap            int64
SMS_received       int64
Assisted            bool
dtype: object

# Change data columns to date type
df['ScheduledDay']=pd.to_datetime(df['ScheduledDay'])
df['AppointmentDay']=pd.to_datetime(df['AppointmentDay'])
df.rename(columns={"ScheduledDay":"ScheduledDate", "AppointmentDay":"AppointmentDate"}, inplace=True)

# Add columns for date day
import datetime
df["AppointmentDay"] = df["AppointmentDate"].dt.strftime("%A")
df["ScheduledDay"] = df["ScheduledDate"].dt.strftime("%A")
df.head(10)

# Add column with the number of days from the Appointment schedule to the appointment date
df["Days to appointment"] =  pd.to_datetime(df["AppointmentDate"]) - pd.to_datetime(df["ScheduledDate"])

# Change column so that only entire days are considered
df["Days to appointment"] = df["Days to appointment"] // np.timedelta64(1, 'D')

df.head()

df["Days to appointment"].describe()

count    110526.000000
mean          9.183794
std          15.255034
min          -7.000000
25%          -1.000000
50%           3.000000
75%          14.000000
max         178.000000
Name: Days to appointment, dtype: float64

fig = px.histogram(df["Days to appointment"])
fig.show()

df[df["Days to appointment"]== -1]

# Replace values
df["Days to appointment"].replace(-1, 0, inplace=True)

df[df["Days to appointment"] < -1]

# Inverse values in columns 1 and 2 (corresponding to ScheduledDate and AppointmentDate) for indexes where the
# Days to appointment value is negative

tmp_values = df[df["Days to appointment"] < -1]["ScheduledDate"].index

for idx in (tmp_values):
    df.iloc[idx,[1,2]] = df.iloc[idx,[2,1]]

# Recalculate the variable Days to appointment 

# Add column with the number of days from the Appointment schedule to the appointment date
df["Days to appointment"] =  pd.to_datetime(df["AppointmentDate"]) - pd.to_datetime(df["ScheduledDate"])

# Change column so that only entire days are considered
df["Days to appointment"] = df["Days to appointment"] // np.timedelta64(1, 'D')

# Replace values
df["Days to appointment"].replace(-1, 0, inplace=True)

fig = px.histogram(df["Days to appointment"])
fig.show()

# Drop full date columns
df.drop(["AppointmentDate","ScheduledDate","Neighbourhood"], axis = 1, inplace=True)
df.head()

df["Assisted"].value_counts()

Assisted
True     88207
False    22319
Name: count, dtype: int64

# Plot Assisted distribution of values
fig = px.histogram(df["Assisted"], title="Number of cases of assistance to the medical appointment",
                   labels={
                       "value": "Assisted"
                   },
                    text_auto=True)
fig.update_traces(showlegend=False)
fig.show()

df_assitance_by_gender = df.reset_index()[["Gender","Assisted"]]
df_assitance_by_gender.head()

df_assitance_by_gender = pd.DataFrame(df_assitance_by_gender.groupby(["Gender","Assisted"]).value_counts())
df_assitance_by_gender.reset_index(inplace=True)
df_assitance_by_gender.rename(columns={0:"count"}, inplace=True)
df_assitance_by_gender

df_assitance_by_gender.groupby(["Gender"])["count"].sum().reset_index()

# Plot the number of appointments made by gender

fig = px.bar(df_assitance_by_gender.groupby(["Gender"])["count"].sum().reset_index(),
            title="Appointments by gender",
            x="Gender", y="count", color="Gender",
            text_auto=True
        )
fig.show()

# Plot the percentage of appointments per gender

fig = px.pie(df_assitance_by_gender.groupby(["Gender"])["count"].sum().reset_index(),
             title = "Appointments distribution by gender",
             color_discrete_map = {
                "F" : "red",
                "M" : "blue"
             },
             names="Gender", values="count"
)
fig.show()

# Plot the appointments assistance by gender 

fig = px.bar(df_assitance_by_gender, barmode="group",
            title ="Appointment assistance by gender",
            color="Gender", y="count", x="Assisted",  text_auto=True)
fig.show()

# Plot assistance by men and women in the same figure

fig = make_subplots(rows=1, cols=2, specs=[[{"type": "pie"}, {"type": "pie"}]])


fig.add_trace(go.Pie(
     values=list(df_assitance_by_gender[df_assitance_by_gender["Gender"]=="M"]["count"]),
     labels=["False", "True"],
     domain=dict(x=[0, 0.5]),
     name="Mens Appointments Assistance"), 
     row=1, col=1)

fig.add_trace(go.Pie(
     values=list(df_assitance_by_gender[df_assitance_by_gender["Gender"]=="F"]["count"]),
     labels=["False", "True"],
     domain=dict(x=[0, 0.5]),
     name="Womens Appointments Assistance"), 
     row=1, col=2)

fig.update_layout(title_text="Assistance to appointments by men and women",
                  height=600,
                  showlegend=True
                  )

fig.show()

# Plot days to appointment distribution based on Assited groups
fig = px.box(df,
             title="Days to appointment for assistance group",
             y="Days to appointment", x="Assisted"
)
fig.show()

# ANOVA test
import scipy.stats as stats

df_days_to_assistance = df[["Assisted","Days to appointment"]]

anova_results = stats.f_oneway(
    df_days_to_assistance["Days to appointment"][df_days_to_assistance["Assisted"] == True],
    df_days_to_assistance["Days to appointment"][df_days_to_assistance["Assisted"] == False]
)

anova_results

F_onewayResult(statistic=3704.465018801432, pvalue=0.0)

df["Age cat"] = pd.cut(df['Age'], [0,3,12,18,26,45,65,116], labels=["Babies","Childs","Adolescents","Young adults","Adults","Mature Adults","Elders"])
age_categories = ["Babies","Childs","Adolescents","Young adults","Adults","Mature Adults","Elders"]

# Plot data distribution based on the age category

fig = px.histogram(df, x=df["Age cat"], color="Age cat",
                    title = "Age distribution of data",
                    text_auto = ".2s",
                    category_orders={ # replaces default order by column name
                        "Age cat": age_categories
                    },
                    labels={
                        "count" : "Number of appointments"
                    }
)

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

# Hide legend
fig.update_layout(showlegend=False)

fig.show()

fig = px.pie(df["Age cat"],
             title = "Age distribution of data",
             names="Age cat"
)
fig.show()

# Create Dataframe with Age category and assistance data
df_assistance_by_age = pd.DataFrame(df.groupby(["Age cat", "Assisted"])["Assisted"].count())
df_assistance_by_age.rename(columns={"Assisted":"Assistance"}, inplace = True)
df_assistance_by_age = df_assistance_by_age.reset_index()

# Add percentage of assistance by Age category
df_assistance_by_age = add_assistance_percentage_per_indicator(df_assistance_by_age,"Age cat")
df_assistance_by_age

# Plot the appointments assistance by age category

fig = px.bar(df_assistance_by_age, barmode="group",
            title ="Appointment assistance by age",
            color="Assisted", y="Assistance", x="Age cat",  text_auto=True)
fig.show()

# Plot the appointments assistance by age category

fig = px.bar(df_assistance_by_age, barmode="group",
            title ="Appointment assistance by age",
            color="Assisted", y="Percentage of appointments",
            labels={
                "Percentage of appointments" : "Percentage of appointments (%)",
                "Age cat": "Age category"
            },
             x="Age cat",  text_auto=True)
fig.show()

fig = px.bar(df_assistance_by_age[df_assistance_by_age["Assisted"] == False ],
            title ="Missing Appointment percentage by age",
            color="Age cat", y="Percentage of appointments", x="Age cat",  text_auto=".4s",
            category_orders={ # replaces default order by column name
                        "Age cat": age_categories
                    })

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

# Hide legend
fig.update_layout(showlegend=False)

fig.show()

# Dataframe creation
df_assistance_by_scholarship = pd.DataFrame(df.groupby(["Scholarship","Assisted"])["Assisted"].count()).rename(columns={"Assisted":"Assistance"}).reset_index()
df_assistance_by_scholarship["Scholarship"] = df_assistance_by_scholarship["Scholarship"].map({0:"No", 1:"Yes"})

# Add percentage of assistance by Scholarship
df_assistance_by_scholarship = add_assistance_percentage_per_indicator(df_assistance_by_scholarship,"Scholarship")
df_assistance_by_scholarship

# Plot appointments per scholarship class
fig = px.bar(df_assistance_by_scholarship,
            barmode="group",
            title="Appointments assistance by scholarship group",
            x="Scholarship", y="Assistance", color="Assisted",
            text_auto=".4s"
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot appointments per scholarship class
fig = px.bar(df_assistance_by_scholarship,
            barmode="group",
            title="Appointments assistance by scholarship group",
            x="Scholarship", y="Percentage of appointments", color="Assisted",
            text_auto=".4s",
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# DataFrame creation with information about the SMS reception and the appointment's assistance
df_assistance_by_sms = pd.DataFrame(df.groupby(["Assisted","SMS_received"])["Assisted"].count()).rename(columns={"Assisted" : "Assistance"}).reset_index()
df_assistance_by_sms["SMS_received"] = df_assistance_by_sms["SMS_received"].map({0:"No", 1:"Yes"})

# Add percentage of assistance by SMS reception
df_assistance_by_sms = add_assistance_percentage_per_indicator(df_assistance_by_sms,"SMS_received")
df_assistance_by_sms

# Plot the number of appointments by SMS reception class
fig = px.bar(df_assistance_by_sms,
            barmode="group",
            title="Appointments assistance by SMS reception",
            x="SMS_received", y="Assistance", color="Assisted",
            text_auto=".4s",
            labels = {
                "Assistance" : "Number of Appointments"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot the number of appointments by SMS reception class
fig = px.bar(df_assistance_by_sms,
            barmode="group",
            title="Appointments assistance by SMS reception",
            x="SMS_received", y="Percentage of appointments", color="Assisted",
            text_auto=".4s",
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)",
                "SMS_received": "SMS received"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot SMS reception variable with assistance and days to appointment for appointments farther than 30 days

fig = px.histogram(df[(df["SMS_received"]== True) & (df["Days to appointment"] > 30)][["Days to appointment","Assisted"]],
    title = "Days to appointment for pattients who receive a SMS",
    color ="Assisted"
)
fig.show()

# Plot SMS reception variable with assistance and days to appointment for appointments farther than 30 days

fig = px.histogram(df[(df["SMS_received"]== False) & (df["Days to appointment"] > 30)][["Days to appointment","Assisted"]],
    title = "Days to appointment for pattients who didn't receive a SMS",
    color ="Assisted"
)
fig.show()

# DataFrame creation with information about the Alcoholism variable and the appointment's assistance
df_assistance_by_alcoholism = pd.DataFrame(df.groupby(["Assisted","Alcoholism"])["Assisted"].count()).rename(columns={"Assisted" : "Assistance"}).reset_index()
df_assistance_by_alcoholism["Alcoholism"] = df_assistance_by_alcoholism["Alcoholism"].map({0:"No", 1:"Yes"})

# Add percentage of assistance by Alcoholism class
df_assistance_by_alcoholism = add_assistance_percentage_per_indicator(df_assistance_by_alcoholism,"Alcoholism")
df_assistance_by_alcoholism

# Plot the number of appointments by Alcoholism group
fig = px.bar(df_assistance_by_alcoholism,
            barmode="group",
            title="Appointments assistance by Alcoholism group",
            x="Alcoholism", y="Assistance", color="Assisted",
            text_auto=".4s",
            labels = {
                "Assistance" : "Number of Appointments"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot the number of appointments by Alcoholism group
fig = px.bar(df_assistance_by_alcoholism,
            barmode="group",
            title="Appointments assistance by Alcoholism group",
            x="Alcoholism", y="Percentage of appointments", color="Assisted",
            text_auto=".4s",
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# DataFrame creation with information about the Diabetes variable and the appointment's assistance
df_assistance_by_diabetes = pd.DataFrame(df.groupby(["Assisted","Diabetes"])["Assisted"].count()).rename(columns={"Assisted" : "Assistance"}).reset_index()
df_assistance_by_diabetes["Alcoholism"] = df_assistance_by_diabetes["Diabetes"].map({0:"No", 1:"Yes"})

# Add percentage of assistance by Diabetes class
df_assistance_by_diabetes = add_assistance_percentage_per_indicator(df_assistance_by_diabetes,"Diabetes")
df_assistance_by_diabetes

# Plot the number of appointments by Diabetes group
fig = px.bar(df_assistance_by_diabetes,
            barmode="group",
            title="Appointments assistance by Diabetes group",
            x="Diabetes", y="Assistance", color="Assisted",
            text_auto=".4s",
            labels = {
                "Assistance" : "Number of Appointments"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot the number of appointments by Diabetes group
fig = px.bar(df_assistance_by_diabetes,
            barmode="group",
            title="Appointments assistance by Diabetes group",
            x="Diabetes", y="Percentage of appointments", color="Assisted",
            text_auto=".4s",
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# DataFrame creation with information about the Hipertension variable and the appointment's assistance
df_assistance_by_hipertension = pd.DataFrame(df.groupby(["Assisted","Hipertension"])["Assisted"].count()).rename(columns={"Assisted" : "Assistance"}).reset_index()
df_assistance_by_hipertension["Hipertension"] = df_assistance_by_hipertension["Hipertension"].map({0:"No", 1:"Yes"})

# Add percentage of assistance by Alcoholism class
df_assistance_by_hipertension = add_assistance_percentage_per_indicator(df_assistance_by_hipertension,"Hipertension")
df_assistance_by_hipertension

# Plot the number of appointments by Hipertension group
fig = px.bar(df_assistance_by_hipertension,
            barmode="group",
            title="Appointments assistance by Hipertension group",
            x="Hipertension", y="Assistance", color="Assisted",
            text_auto=".4s",
            labels = {
                "Assistance" : "Number of Appointments"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

# Plot the number of appointments by Hipertension group
fig = px.bar(df_assistance_by_hipertension,
            barmode="group",
            title="Appointments assistance by Hipertension group",
            x="Hipertension", y="Percentage of appointments", color="Assisted",
            text_auto=".4s",
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)"
            }
        )

# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

fig.show()

days = ["Monday", "Tuesday", "Wednesday","Thursday","Friday","Saturday","Sunday"]

# Plot number of appointments by the day they were made

fig = px.histogram(df["AppointmentDay"], 
                title = "Appointmends made by day",
                labels = {
                    "value" : "Day",
                    "count" : "Appointments"
                },
                category_orders={ # replaces default order by column name
                "value": days
            }, color="value",
            text_auto=".4s")


# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

# Hide legend
fig.update_layout(showlegend=False)

fig.show()

# Plot number of appointments by the day they were scheduled

fig = px.histogram(df["ScheduledDay"], 
                title = "Appointmends scheduled by day",
                labels = {
                    "value" : "Day",
                    "count" : "Appointments"
                },
                category_orders={ # replaces default order by column name
                "value": days
            }, color="value",
            text_auto=".4s")


# Set values over the bars
fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)

# Hide legend
fig.update_layout(showlegend=False)

fig.show()

# Create a Dataframe with appointmend day assistance information
df_assistance_by_appointmentday = pd.DataFrame(df.groupby(["AppointmentDay","Assisted"])["Assisted"].count()).rename(columns={"Assisted":"Assistance"}).reset_index()
df_assistance_by_appointmentday

# Plot the appointments assistance by appointment's made day

fig = px.bar(df_assistance_by_appointmentday, barmode="group",
            title ="Appointment assistance by Appointments made day",
            color="Assisted", y="Assistance", x="AppointmentDay",  text_auto=True,
            category_orders={
                "AppointmentDay": days
            },
            labels = {
                "Assistance" : "Number of appointments"
            }
)
fig.show()

# Add percentage of assistance by Appointments made day
df_assistance_by_appointmentday = add_assistance_percentage_per_indicator(df_assistance_by_appointmentday,"AppointmentDay")
df_assistance_by_appointmentday

# Plot the appointments assistance by appointment's made day

fig = px.bar(df_assistance_by_appointmentday, barmode="group",
            title ="Appointment assistance by Appointments made day",
            color="Assisted", y="Percentage of appointments", x="AppointmentDay",  text_auto=True,
            category_orders={
                "AppointmentDay": days
            },
            labels = {
                "Percentage of appointments" : "Percentage of appointments (%)"
            }
)
fig.show()

# Plot missed assistance per Appointment made day
plot_missing_assistance_percentage_per_indicator(df_assistance_by_appointmentday, "AppointmentDay",
                                                 "Missing Appointment percentage by Appointment made Day")

# Create a Dataframe with scheduled day assistance information
df_assistance_by_scheduledday = pd.DataFrame(df.groupby(["ScheduledDay","Assisted"])["Assisted"].count()).rename(columns={"Assisted":"Assistance"}).reset_index()
df_assistance_by_scheduledday

# Plot the appointments assistance by appointment's scheduled day

fig = px.bar(df_assistance_by_scheduledday, barmode="group",
            title ="Appointment assistance by Appointments scheduled day",
            color="Assisted", y="Assistance", x="ScheduledDay",  text_auto=True,
            category_orders={
                "ScheduledDay": days
            },
            labels={
                "Assistance":"Number of appointments"
            }
)
fig.show()

# Add percentage of assistance by Appointments scheduled day
df_assistance_by_scheduledday = add_assistance_percentage_per_indicator(df_assistance_by_scheduledday,"ScheduledDay")
df_assistance_by_scheduledday

# Plot the appointments assistance by appointment's scheduled day

fig = px.bar(df_assistance_by_scheduledday, barmode="group",
            title ="Appointment assistance by scheduled day",
            color="Assisted", y="Percentage of appointments", x="ScheduledDay",  text_auto=True,
            category_orders={
                "ScheduledDay": days
            },
            labels={
                "Percentage of appointments":"Percentage of appointments (%)"
            }
)
fig.show()

# Plot missed assistance per Appointment's scheduled day
plot_missing_assistance_percentage_per_indicator(df_assistance_by_scheduledday, "ScheduledDay",
                                                 "Missing Appointment percentage by Scheduled day")

from sklearn.model_selection import train_test_split

data = df.copy()
data.drop(["Age"], axis=1, inplace=True)
data.head()

# Create dummy variables for categorical variables
data = pd.get_dummies(data)
data.head()

# Create train and test splits
Y = data["Assisted"]
X = data.drop(["Assisted"], axis=1)
x_train, x_test, y_train, y_test = train_test_split(X,Y, test_size=0.3, random_state = 0)
print("Train dataset size ",x_train.shape[0])
print("Test dataset size ",x_test.shape[0])

Train dataset size  77368
Test dataset size  33158

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix

# Simple model test 
lr = LogisticRegression(C= 0.01, solver="newton-cg", class_weight = {0:0.65, 1:0.35})
lr.fit(x_train, y_train)

LogisticRegression(C=0.01, class_weight={0: 0.65, 1: 0.35}, solver='newton-cg')

LogisticRegression(C=0.01, class_weight={0: 0.65, 1: 0.35}, solver='newton-cg')

# Predictions of the simple model
y_hat = lr.predict(x_test)

# Logistic regression score
lr_score = lr.score(x_test, y_test)
lr_score

0.7768864225827854

# Confusion Matrix
cnf_matrix = confusion_matrix(y_test, y_hat, labels=[1,0])
cnf_matrix

array([[24992,  1573],
       [ 5825,   768]], dtype=int64)

# Plot confusion matrix
plot_confusion_matrix(cnf_matrix, title="Confusion matrix")

# Plot ROC Curve
plot_roc_curve(x_test, y_test, lr)

!pip install graphviz

Collecting graphviz
  Downloading graphviz-0.20.3-py3-none-any.whl.metadata (12 kB)
Downloading graphviz-0.20.3-py3-none-any.whl (47 kB)
   ---------------------------------------- 0.0/47.1 kB ? eta -:--:--
   -------- ------------------------------- 10.2/47.1 kB ? eta -:--:--
   ----------------- ---------------------- 20.5/47.1 kB 682.7 kB/s eta 0:00:01
   ----------------- ---------------------- 20.5/47.1 kB 682.7 kB/s eta 0:00:01
   ----------------- ---------------------- 20.5/47.1 kB 682.7 kB/s eta 0:00:01
   ---------------------------------------- 47.1/47.1 kB 236.4 kB/s eta 0:00:00
Installing collected packages: graphviz
Successfully installed graphviz-0.20.3

from sklearn.tree import DecisionTreeClassifier
from sklearn import metrics
import graphviz

# Create Decision Tree classifer object
dt = DecisionTreeClassifier(class_weight="balanced")

# Train Decision Tree Classifer
dt = dt.fit(x_train,y_train)

#Predict the response for test dataset
y_hat = dt.predict(x_test)

# Model Accuracy
print("Accuracy:",metrics.accuracy_score(y_test, y_hat))

Accuracy: 0.6428916098679052

# Confusion Matrix
cnf_matrix = confusion_matrix(y_test, y_hat, labels=[0,1])

# Plot confusion matrix
plot_confusion_matrix(cnf_matrix, title="Confusion matrix")

# Plot ROC Curve
plot_roc_curve(x_test, y_test, dt)

	PatientId	AppointmentID	Gender	ScheduledDay	AppointmentDay	Age	Neighbourhood	Hipertension	Diabetes	No-show
0	2.987250e+13	5642903	F	2016-04-29T18:38:08Z	2016-04-29T00:00:00Z	62	JARDIM DA PENHA	1	0	No
1	5.589978e+14	5642503	M	2016-04-29T16:08:27Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	0	0	No
2	4.262962e+12	5642549	F	2016-04-29T16:19:04Z	2016-04-29T00:00:00Z	62	MATA DA PRAIA	0	0	No
3	8.679512e+11	5642828	F	2016-04-29T17:29:31Z	2016-04-29T00:00:00Z	8	PONTAL DE CAMBURI	0	0	No
4	8.841186e+12	5642494	F	2016-04-29T16:07:23Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	1	1	No

	PatientId	AppointmentID	Age	Scholarship	Hipertension	Diabetes	Alcoholism	Handcap	SMS_received
count	1.105270e+05	1.105270e+05	110527.000000	110527.000000	110527.000000	110527.000000	110527.000000	110527.000000	110527.000000
mean	1.474963e+14	5.675305e+06	37.088874	0.098266	0.197246	0.071865	0.030400	0.022248	0.321026
std	2.560949e+14	7.129575e+04	23.110205	0.297675	0.397921	0.258265	0.171686	0.161543	0.466873
min	3.921784e+04	5.030230e+06	-1.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	4.172614e+12	5.640286e+06	18.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
50%	3.173184e+13	5.680573e+06	37.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
75%	9.439172e+13	5.725524e+06	55.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000
max	9.999816e+14	5.790484e+06	115.000000	1.000000	1.000000	1.000000	1.000000	4.000000	1.000000

	PatientId	AppointmentID	Age	Scholarship	Hipertension	Diabetes	Alcoholism	Handcap	SMS_received
count	1.105260e+05	1.105260e+05	110526.000000	110526.000000	110526.000000	110526.000000	110526.000000	110526.000000	110526.000000
mean	1.474934e+14	5.675304e+06	37.089219	0.098266	0.197248	0.071865	0.030400	0.022248	0.321029
std	2.560943e+14	7.129544e+04	23.110026	0.297676	0.397923	0.258266	0.171686	0.161543	0.466874
min	3.921784e+04	5.030230e+06	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	4.172536e+12	5.640285e+06	18.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
50%	3.173184e+13	5.680572e+06	37.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
75%	9.438963e+13	5.725523e+06	55.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000
max	9.999816e+14	5.790484e+06	115.000000	1.000000	1.000000	1.000000	1.000000	4.000000	1.000000

	Gender	ScheduledDay	AppointmentDay	Age	Neighbourhood	Hipertension	Diabetes	No-show
0	F	2016-04-29T18:38:08Z	2016-04-29T00:00:00Z	62	JARDIM DA PENHA	1	0	No
1	M	2016-04-29T16:08:27Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	0	0	No
2	F	2016-04-29T16:19:04Z	2016-04-29T00:00:00Z	62	MATA DA PRAIA	0	0	No
3	F	2016-04-29T17:29:31Z	2016-04-29T00:00:00Z	8	PONTAL DE CAMBURI	0	0	No
4	F	2016-04-29T16:07:23Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	1	1	No

	Gender	ScheduledDay	AppointmentDay	Age	Neighbourhood	Hipertension	Diabetes	Assisted
0	F	2016-04-29T18:38:08Z	2016-04-29T00:00:00Z	62	JARDIM DA PENHA	1	0	True
1	M	2016-04-29T16:08:27Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	0	0	True
2	F	2016-04-29T16:19:04Z	2016-04-29T00:00:00Z	62	MATA DA PRAIA	0	0	True
3	F	2016-04-29T17:29:31Z	2016-04-29T00:00:00Z	8	PONTAL DE CAMBURI	0	0	True
4	F	2016-04-29T16:07:23Z	2016-04-29T00:00:00Z	56	JARDIM DA PENHA	1	1	True

Medical Appointment Analysis¶

Dataset Description¶

Data Dictionary¶

EDA Questions¶

Setup¶

Defined functions¶

Dataset Overview¶

Loading the dataset and checking the columns we have¶

Data cleaning¶

Exploratory Data Analysis (EDA)¶

Gender analysis¶

Days to appointment Analysis¶

Age analaysis¶

Scholarship Analysis¶

SMS reception Analysis¶

Alcoholism Analysis¶

Diabetes Analysis¶

Hipertension Analysis¶

Scheduled day and Appointment day Analysis¶

Data pre-processing¶

Logistic Regression Model¶

Decision tree classifier¶

Conclusions¶

	0
PatientId	float64
AppointmentID	int64
Gender	object
ScheduledDay	object
AppointmentDay	object
Age	int64
Neighbourhood	object
Scholarship	int64
Hipertension	int64
Diabetes	int64
Alcoholism	int64
Handcap	int64
SMS_received	int64
No-show	object

	Gender	ScheduledDate	AppointmentDate	Age	Neighbourhood	Hipertension	Diabetes	Assisted	AppointmentDay	ScheduledDay
0	F	2016-04-29 18:38:08+00:00	2016-04-29 00:00:00+00:00	62	JARDIM DA PENHA	1	0	True	Friday	Friday
1	M	2016-04-29 16:08:27+00:00	2016-04-29 00:00:00+00:00	56	JARDIM DA PENHA	0	0	True	Friday	Friday
2	F	2016-04-29 16:19:04+00:00	2016-04-29 00:00:00+00:00	62	MATA DA PRAIA	0	0	True	Friday	Friday
3	F	2016-04-29 17:29:31+00:00	2016-04-29 00:00:00+00:00	8	PONTAL DE CAMBURI	0	0	True	Friday	Friday
4	F	2016-04-29 16:07:23+00:00	2016-04-29 00:00:00+00:00	56	JARDIM DA PENHA	1	1	True	Friday	Friday
5	F	2016-04-27 08:36:51+00:00	2016-04-29 00:00:00+00:00	76	REPÚBLICA	1	0	True	Friday	Wednesday
6	F	2016-04-27 15:05:12+00:00	2016-04-29 00:00:00+00:00	23	GOIABEIRAS	0	0	False	Friday	Wednesday
7	F	2016-04-27 15:39:58+00:00	2016-04-29 00:00:00+00:00	39	GOIABEIRAS	0	0	False	Friday	Wednesday
8	F	2016-04-29 08:02:16+00:00	2016-04-29 00:00:00+00:00	21	ANDORINHAS	0	0	True	Friday	Friday
9	F	2016-04-27 12:48:25+00:00	2016-04-29 00:00:00+00:00	19	CONQUISTA	0	0	True	Friday	Wednesday

	Gender	ScheduledDate	AppointmentDate	Age	Neighbourhood	Handcap	Assisted	AppointmentDay	ScheduledDay	Days to appointment
27033	M	2016-05-10 10:51:53+00:00	2016-05-09 00:00:00+00:00	38	RESISTÊNCIA	1	False	Monday	Tuesday	-2
55226	F	2016-05-18 14:50:41+00:00	2016-05-17 00:00:00+00:00	19	SANTO ANTÔNIO	1	False	Tuesday	Wednesday	-2
64175	F	2016-05-05 13:43:58+00:00	2016-05-04 00:00:00+00:00	22	CONSOLAÇÃO	0	False	Wednesday	Thursday	-2
71533	F	2016-05-11 13:49:20+00:00	2016-05-05 00:00:00+00:00	81	SANTO ANTÔNIO	0	False	Thursday	Wednesday	-7
72362	M	2016-05-04 06:50:57+00:00	2016-05-03 00:00:00+00:00	7	TABUAZEIRO	0	False	Tuesday	Wednesday	-2

	Age cat	Assisted	Assistance	Total Age cat appointments	Percentage of appointments
0	Babies	False	944	5404	17.468542
1	Babies	True	4460	5404	82.531458
2	Childs	False	2724	12093	22.525428
3	Childs	True	9369	12093	77.474572
4	Adolescents	False	2040	7830	26.053640
5	Adolescents	True	5790	7830	73.946360
6	Young adults	False	2783	11016	25.263253
7	Young adults	True	8233	11016	74.736747
8	Adults	False	6052	27703	21.846009
9	Adults	True	21651	27703	78.153991
10	Mature Adults	False	5071	29640	17.108637
11	Mature Adults	True	24569	29640	82.891363
12	Elders	False	2066	13301	15.532667
13	Elders	True	11235	13301	84.467333

	Scholarship	Assisted	Assistance	Total Scholarship appointments	Percentage of appointments
0	No	False	19741	99665	19.807355
1	No	True	79924	99665	80.192645
2	Yes	False	2578	10861	23.736304
3	Yes	True	8283	10861	76.263696

	Assisted	SMS_received	Assistance	Total SMS_received appointments	Percentage of appointments
0	False	No	12535	75044	16.703534
1	False	Yes	9784	35482	27.574545
2	True	No	62509	75044	83.296466
3	True	Yes	25698	35482	72.425455

	Assisted	Alcoholism	Assistance	Total Alcoholism appointments	Percentage of appointments
0	False	No	21642	107166	20.194838
1	False	Yes	677	3360	20.148810
2	True	No	85524	107166	79.805162
3	True	Yes	2683	3360	79.851190

	Assisted	Diabetes	Assistance	Alcoholism	Total Diabetes appointments	Percentage of appointments
0	False	0	20889	No	102583	20.363023
1	False	1	1430	Yes	7943	18.003273
2	True	0	81694	No	102583	79.636977
3	True	1	6513	Yes	7943	81.996727

	Assisted	Hipertension	Assistance	Total Hipertension appointments	Percentage of appointments
0	False	No	18547	88725	20.903917
1	False	Yes	3772	21801	17.301959
2	True	No	70178	88725	79.096083
3	True	Yes	18029	21801	82.698041

	AppointmentDay	Assisted	Assistance
0	Friday	False	4037
1	Friday	True	14982
2	Monday	False	4690
3	Monday	True	18024
4	Saturday	False	9
5	Saturday	True	30
6	Thursday	False	3338
7	Thursday	True	13909
8	Tuesday	False	5152
9	Tuesday	True	20488
10	Wednesday	False	5093
11	Wednesday	True	20774

	Gender	count
0	F	71839
1	M	38687

	ScheduledDay	Assisted	Assistance	Total ScheduledDay appointments	Percentage of appointments
0	Friday	False	3887	18915	20.549828
1	Friday	True	15028	18915	79.450172
2	Monday	False	4561	23084	19.758274
3	Monday	True	18523	23084	80.241726
4	Saturday	False	1	24	4.166667
5	Saturday	True	23	24	95.833333
6	Thursday	False	3700	18073	20.472528
7	Thursday	True	14373	18073	79.527472
8	Tuesday	False	5291	26168	20.219352
9	Tuesday	True	20877	26168	79.780648
10	Wednesday	False	4879	24262	20.109636
11	Wednesday	True	19383	24262	79.890364